{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-22T10:21:45.053373Z",
     "start_time": "2025-01-22T10:21:45.050057Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "import math"
   ],
   "id": "cf3c301704bee1cd",
   "outputs": [],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-22T09:51:10.763965Z",
     "start_time": "2025-01-22T09:51:10.666964Z"
    }
   },
   "cell_type": "code",
   "source": [
    "random = torch.randn(1, 10, 512)\n",
    "print(random)"
   ],
   "id": "472fb5d7bfdf6f5e",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[ 0.5600, -1.9831,  0.6232,  ..., -0.7687, -0.5738,  0.2720],\n",
      "         [-0.3855,  0.3229,  0.5258,  ..., -0.3466, -0.3972, -1.1322],\n",
      "         [ 0.1405, -0.6451, -0.3845,  ..., -0.2641, -1.5250, -0.7047],\n",
      "         ...,\n",
      "         [-0.1460,  0.1705,  0.6061,  ...,  1.1041, -0.0763,  0.3173],\n",
      "         [ 0.4749, -1.6067, -1.0120,  ..., -0.6799,  0.0513, -0.0940],\n",
      "         [-0.2678, -1.2888,  0.6297,  ..., -1.5349,  1.3751,  0.4755]]])\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-22T10:21:47.270435Z",
     "start_time": "2025-01-22T10:21:47.265649Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from torch import Tensor\n",
    "# 将输入的词汇表转化为指定维度的Embedding\n",
    "\n",
    "class TokenEmbedding(nn.Embedding):\n",
    "    def __int__(self,vocab_size,d_model):\n",
    "        super(TokenEmbedding,self).__init__(vocab_size,d_model,padding_idx=1)"
   ],
   "id": "3be0da3de959d9b8",
   "outputs": [],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-22T10:21:48.833415Z",
     "start_time": "2025-01-22T10:21:48.827368Z"
    }
   },
   "cell_type": "code",
   "source": [
    "\n",
    " \"\"\"\n",
    "    实现位置嵌入（Positional Embedding），用于给输入序列添加位置信息。\n",
    "\n",
    "    参数:\n",
    "    d_model (int): 嵌入向量的维度。\n",
    "    max_len (int): 序列的最大长度。\n",
    "    device (torch.device): 运算设备，CPU或GPU。\n",
    "    \"\"\"\n",
    "class PositionalEmbedding(nn.Module):\n",
    "    def __init__(self,d_model,max_len,device):\n",
    "        super(PositionalEmbedding,self).__init__()\n",
    "        # 位置索引向量\n",
    "        self.encoding = torch.zeros(max_len,d_model,device=device)\n",
    "        # 不对位置编码进行梯度更新\n",
    "        self.encoding.requires_grad = False\n",
    "        # 创建位置索引\n",
    "        pos=torch.arange(0,max_len,device=device)\n",
    "        pos=pos.float().unsqueeze(dim=1)\n",
    "        # 计算2i的位置\n",
    "        _2i=torch.arange(0,d_model,step=2,device=device).float()\n",
    "        # 使用正弦函数编码偶数维度的位置信息\n",
    "        self.encoding[:,0::2]=torch.sin(pos/10000**(_2i/d_model))\n",
    "        # 使用余弦函数编码奇数维度的位置信息\n",
    "        self.encoding[:,1::2]=torch.cos(pos/10000**(_2i/d_model))\n",
    "\n",
    "    def forward(self,x):\n",
    "        batch_size,seq_len=x.size()\n",
    "        return self.encoding[:seq_len,:]"
   ],
   "id": "bcc87f59fc5cdd74",
   "outputs": [],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-22T10:21:50.740685Z",
     "start_time": "2025-01-22T10:21:50.735743Z"
    }
   },
   "cell_type": "code",
   "source": [
    "class TransformerEmbedding(nn.Module):\n",
    "    def __init__(self,vocab_size,d_model,max_len,drop_prob,device):\n",
    "        super(TransformerEmbedding,self).__init__()\n",
    "        self.token_embedding=TokenEmbedding(vocab_size,d_model)\n",
    "        self.position_embedding=PositionalEmbedding(d_model,max_len,device)\n",
    "        self.dropout=nn.Dropout(p=drop_prob)\n",
    "\n",
    "    def forward(self,x):\n",
    "        # 获取输入的词汇表\n",
    "        x=self.token_embedding(x)\n",
    "        # 获取位置信息\n",
    "        pos=self.position_embedding(x)\n",
    "        # 将词汇表和位置信息相加\n",
    "        x=x+pos\n",
    "        # 对输入进行dropout处理\n",
    "        x=self.dropout(x)\n",
    "        return x\n"
   ],
   "id": "d2290fd85a4b3aef",
   "outputs": [],
   "execution_count": 7
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "3ded1479af82f428"
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "language": "python",
   "display_name": "Python 3 (ipykernel)"
  }
 },
 "nbformat": 5,
 "nbformat_minor": 9
}
